from langchain_community.document_loaders import UnstructuredMarkdownLoader
from langchain_text_splitters import RecursiveCharacterTextSplitter

# 构建Markdown 文档加载器并获取文档列表
# LLMOps 项目 API 文档
loader = UnstructuredMarkdownLoader("./project_api.md")
documents = loader.load()

# 构建分割器
text_splitter = RecursiveCharacterTextSplitter(
    separators=[
        "\n\n",
        "\n",
        "。|！|？",
        "\.\s|\!\s|\?\s",  # 英文标点符号后面通常需要加空格
        "；|;\s",
        "，|,\s",
        " ",
        ""
    ],
    chunk_size=500,
    chunk_overlap=50,
    add_start_index=True
)

# 分割文档列表
chunks = text_splitter.split_documents(documents)

# 输出信息
for chunk in chunks:
    print(f"块大小{len(chunk.page_content)},元数据{chunk.metadata}")